package engine

import (
	"gitee.com/goTest/cranwler/fetcher"
	"log"
)

func Run(seeds ...Request) {
	var reqs []Request

	for _, r := range seeds {
		reqs = append(reqs, r)
	}

	for len(reqs) > 0 {
		r := reqs[0]

		reqs = reqs[1:]

		log.Printf("获取：%s", r.Url)
		body, err := fetcher.Fetch(r.Url)
		if err != nil {
			log.Printf("Fatcher error")
			continue
		}

		parserResult := r.ParserFun(body)

		reqs = append(reqs, parserResult.Request...)

		for _, item := range parserResult.Items {
			log.Printf(" Got items %v", item)
		}
	}

}
